In [10]:
import numpy as np
import pandas as pd
import json
import sys
import os
import matplotlib
matplotlib.use('Agg') 
import matplotlib.pyplot as plt
import seaborn as sns
import pdb

from util import utils as data_utils

%pylab inline
%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'Blues'

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

json_file = './cifar_results/noise/bootstrap_const_lr_001/checkpoint_200.json'
FDIR = os.path.dirname(json_file)
NUM_CLASSIFY = 10
Populating the interactive namespace from numpy and matplotlib
The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
/usr/local/lib/python2.7/dist-packages/matplotlib/__init__.py:1350: UserWarning:  This call to matplotlib.use() has no effect
because the backend has already been chosen;
matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.

  warnings.warn(_use_error_msg)
In [ ]:
 
In [11]:
# Plot gradients norms for the entire learning process
grads_json_filename = os.path.join(FDIR, 'model_grads.json')
grads = [[], [], []]
grads_key = ['max_grad_w1_16', 'max_grad_w1_32', 'max_grad_w1_64']
if os.path.exists(grads_json_filename):
    with open(grads_json_filename, 'r') as fp:
        data = json.load(fp)
        for i, k in enumerate(grads_key):
            if data[0].get(k, None) is None:
                continue
            for batch_grads in data:
                grads[i].append(batch_grads[k])

def plot_grads(grads, title, x_label, y_label, figsize=(10, 8)):
    plt.figure(figsize=figsize)
    # plt.subplot(2, 1, 1)
    plt.plot(grads)
    plt.title(title)
    plt.ylabel(y_label)
    plt.xlabel(x_label)
    
for i, g in enumerate(grads):
    if len(g) > 0:
        plot_grads(g, grads_key[i], 'iterations', grads_key[i])
        # pass
In [12]:
with open(json_file, 'r') as fp:
    data = json.load(fp)
# Loss history might not be of equal length.
train_loss_hist = data['train_loss_history']
val_loss_hist = data['val_loss_history']

# pdb.set_trace()
def plot_loss_hist(loss_hist, title,):
    plt.figure(figsize=(5,4))
    plt.subplot(1, 1, 1)
    plt.plot(loss_hist)
    plt.title(title)  # Train Loss
    plt.ylabel('loss')
    plt.xlabel('time')
    plt.show()
    
plot_loss_hist(train_loss_hist, 'Train Loss')
plot_loss_hist(val_loss_hist, 'Val loss')

if data.get('crit1_loss_history', None) is not None:
    plot_loss_hist(data['crit1_loss_history'], 'Target criterion loss')

if data.get('crit2_loss_history', None) is not None and \
    len(data['crit2_loss_history']) > 0:
    plot_loss_hist(data['crit2_loss_history'], 'Pred criterion loss')

if data.get('pred_loss_history', None) is not None and \
    len(data['pred_loss_history']) > 0:
    plot_loss_hist(data['pred_loss_history'], 'Total Pred loss (beta*t + (1-beta)*p)')    

if data.get('beta_loss_history', None) is not None and \
    len(data['beta_loss_history']) > 0:
    plot_loss_hist(data['beta_loss_history'], 'Beta loss')
In [13]:
if data.get('KL_loss_history', None) is not None:
    # Loss history might not be of equal length.
    KL_loss_hist = data['KL_loss_history']

    plt.figure(figsize=(10,8))
    plt.plot(KL_loss_hist)
    plt.title('KL loss')
    plt.ylabel('loss')
    plt.xlabel('time')
    plt.show()
In [14]:
def get_conf(json_file, num_classes=26, json_key='conf'):
    with open(json_file, 'r') as fp:
        data = json.load(fp)
        conf = data.get(json_key, None)
    if conf is None:
        return
    # c1 = conf.split('\n')[1].split("]")[0].split("[ ")[1].split(" ")
    c1 = conf.split('\n')
    # print(c1)
    conf_mat, row_idx = np.zeros((num_classes, num_classes)), 0
    for i in c1:
        #pdb.set_trace()
        is_conf_row = False
        if ']' in i and '[[' in i:
            val = i.split(']')[0].split('[[')[1].split(' ')
            is_conf_row = True
        elif ']' in i and '[' in i:
            val = i.split(']')[0].split('[')[1].split(' ')
            is_conf_row = True
        if is_conf_row:
            col_idx = 0
            for v in val:
                if not len(v):
                    continue
                try:
                    conf_mat[row_idx, col_idx] = int(v)
                    col_idx = col_idx + 1
                except:
                    continue
            row_idx = row_idx + 1
    
    assert(row_idx == num_classes)
    conf_mat = conf_mat.astype(int)
    fdir = os.path.dirname(json_file)
    json_name = os.path.basename(json_file)[:-5]
    conf_file_name = fdir + '/' + 'conf_' + json_name + '.txt'
    np.savetxt(conf_file_name, conf_mat, fmt='%d', delimiter=', ')
    return conf_mat


def plot_conf(norm_conf):
  # Plot using seaborn
  # (this is style I used for ResNet matrix)
  plt.figure(figsize=(10,6))
  df_cm = pd.DataFrame(norm_conf)
  sns.heatmap(df_cm, annot=True, cmap="Blues")
  plt.show()
In [15]:
def get_sorted_checkpoints(fdir):
    # Checkpoint files are named as 'checkpoint_%d.json'
    checkpoint_map = {}
    for f in os.listdir(fdir):
        if f.endswith('json') and f.startswith('checkpoint'):
            checkpoint_num = int(f.split('checkpoint_')[-1].split('.')[0])
            checkpoint_map[checkpoint_num] = f
    sorted_checkpoints = []
    for k in sorted(checkpoint_map.keys()):
        v = checkpoint_map[k]
        sorted_checkpoints.append(v)
    return sorted_checkpoints
In [16]:
def best_f_scores(fdir, num_classes=5): 
    best_checkpoints = [None, None, None]
    best_3_fscores = [0, 0, 0]
    best_confs = [np.array(()), np.array(()), np.array(())]
    f1_weight_list = [1.0] * num_classes
    f1_weights = np.array(f1_weight_list)
    sorted_checkpoint_files = get_sorted_checkpoints(fdir)
    for f in sorted_checkpoint_files:
        json_file = fdir + '/' + f
        conf = get_conf(json_file, num_classes, json_key='val_conf')
        norm_conf = data_utils.normalize_conf(conf)
        f1 = data_utils.get_f1_score(conf, f1_weights)
        kappa = data_utils.computeKappa(conf)
        wt_f1 = data_utils.computeWeightedF1(conf)
        print('file: {}, f1: {:.3f}, kappa: {:.3f}, weighted-F1: {:.3f}'.format(
                f, f1, kappa, wt_f1))
        plot_conf(norm_conf)
        max_idx = -1
        for i in range(len(best_3_fscores)):
            if best_3_fscores[i] > f1:
                break
            max_idx = i
        for j in range(max_idx):
            best_3_fscores[j] = best_3_fscores[j+1]
            best_confs[j] = best_confs[j+1]
            best_checkpoints[j] = best_checkpoints[j+1]

        best_3_fscores[max_idx] = f1
        best_confs[max_idx] = conf
        best_checkpoints[max_idx] = f

    return best_3_fscores, best_confs, best_checkpoints
In [17]:
def plot_train_conf(fdir, num_classes=5):
    sorted_checkpoint_files = get_sorted_checkpoints(fdir)
    if len(sorted_checkpoint_files) > 0:
        last_checkpoint = sorted_checkpoint_files[-1]
        json_file = fdir + '/' + last_checkpoint
        conf = get_conf(json_file, num_classes=num_classes, json_key='train_conf')
        print(conf)
        norm_conf = data_utils.normalize_conf(conf)
        f1_weight_list = [1.0] * num_classes
        f1_weights = np.array(f1_weight_list)
        f1 = data_utils.get_f1_score(conf, f1_weights)
        kappa = data_utils.computeKappa(conf)
        wt_f1 = data_utils.computeWeightedF1(conf)
        print('file: {}, f1: {:.3f}, kappa: {:.3f}, weighted-F1: {:.3f}'.format(
            f, f1, kappa, wt_f1))
        plot_conf(norm_conf)

plot_train_conf(FDIR, num_classes=NUM_CLASSIFY)
[[3191   18  126   47   46   17 1441   16   75   46]
 [  35 3528    1   11    2    2    2    7   33 1338]
 [  73    2 2881  235  100 1530   66   53    6   19]
 [  36   19   66 2926   77  359   70   28 1434   29]
 [  16    1   75   93 3229   86   72 1398    7   11]
 [  29    1 1285  395  108 3088   37   76    4   12]
 [  30   11  100  110 1360   62 3239   35    1    1]
 [  17 1458   26   48   97   66    2 3260    9   29]
 [  76   24   34 1224   29  150   26   23 3345   25]
 [1422  203   43   22   12    5    9   14   65 3274]]
file: <built-in method f of mtrand.RandomState object at 0x7f616712f5f0>, f1: 0.639, kappa: 0.421, weighted-F1: 0.639
In [18]:
best_f_scores(FDIR, num_classes=NUM_CLASSIFY)
file: checkpoint_1.json, f1: 0.170, kappa: -0.033, weighted-F1: 0.170
file: checkpoint_2.json, f1: 0.277, kappa: 0.071, weighted-F1: 0.277
file: checkpoint_3.json, f1: 0.492, kappa: 0.530, weighted-F1: 0.492
file: checkpoint_4.json, f1: 0.570, kappa: 0.558, weighted-F1: 0.570
file: checkpoint_5.json, f1: 0.646, kappa: 0.631, weighted-F1: 0.646
file: checkpoint_6.json, f1: 0.726, kappa: 0.731, weighted-F1: 0.726
file: checkpoint_7.json, f1: 0.744, kappa: 0.745, weighted-F1: 0.744
file: checkpoint_8.json, f1: 0.725, kappa: 0.656, weighted-F1: 0.725
file: checkpoint_9.json, f1: 0.778, kappa: 0.786, weighted-F1: 0.778
file: checkpoint_10.json, f1: 0.794, kappa: 0.803, weighted-F1: 0.794
file: checkpoint_11.json, f1: 0.778, kappa: 0.782, weighted-F1: 0.778
file: checkpoint_12.json, f1: 0.799, kappa: 0.799, weighted-F1: 0.799
file: checkpoint_13.json, f1: 0.799, kappa: 0.802, weighted-F1: 0.799
file: checkpoint_14.json, f1: 0.813, kappa: 0.814, weighted-F1: 0.813
file: checkpoint_15.json, f1: 0.813, kappa: 0.803, weighted-F1: 0.813
file: checkpoint_16.json, f1: 0.816, kappa: 0.801, weighted-F1: 0.816
file: checkpoint_17.json, f1: 0.827, kappa: 0.826, weighted-F1: 0.827
file: checkpoint_18.json, f1: 0.827, kappa: 0.823, weighted-F1: 0.827
file: checkpoint_19.json, f1: 0.816, kappa: 0.811, weighted-F1: 0.816
file: checkpoint_20.json, f1: 0.840, kappa: 0.841, weighted-F1: 0.840
file: checkpoint_21.json, f1: 0.823, kappa: 0.817, weighted-F1: 0.823
file: checkpoint_22.json, f1: 0.832, kappa: 0.836, weighted-F1: 0.832
file: checkpoint_23.json, f1: 0.841, kappa: 0.850, weighted-F1: 0.841
file: checkpoint_24.json, f1: 0.853, kappa: 0.840, weighted-F1: 0.853
file: checkpoint_25.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_26.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_27.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_28.json, f1: 0.858, kappa: 0.854, weighted-F1: 0.858
file: checkpoint_29.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_30.json, f1: 0.853, kappa: 0.843, weighted-F1: 0.853
file: checkpoint_31.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_32.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_33.json, f1: 0.853, kappa: 0.845, weighted-F1: 0.853
file: checkpoint_34.json, f1: 0.856, kappa: 0.852, weighted-F1: 0.856
file: checkpoint_35.json, f1: 0.853, kappa: 0.847, weighted-F1: 0.853
file: checkpoint_36.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_37.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_38.json, f1: 0.856, kappa: 0.846, weighted-F1: 0.856
file: checkpoint_39.json, f1: 0.856, kappa: 0.846, weighted-F1: 0.856
file: checkpoint_40.json, f1: 0.854, kappa: 0.848, weighted-F1: 0.854
file: checkpoint_41.json, f1: 0.854, kappa: 0.848, weighted-F1: 0.854
file: checkpoint_42.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_43.json, f1: 0.856, kappa: 0.852, weighted-F1: 0.856
file: checkpoint_44.json, f1: 0.855, kappa: 0.844, weighted-F1: 0.855
file: checkpoint_45.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_46.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_47.json, f1: 0.857, kappa: 0.853, weighted-F1: 0.857
file: checkpoint_48.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_49.json, f1: 0.856, kappa: 0.847, weighted-F1: 0.856
file: checkpoint_50.json, f1: 0.856, kappa: 0.845, weighted-F1: 0.856
file: checkpoint_51.json, f1: 0.857, kappa: 0.848, weighted-F1: 0.857
file: checkpoint_52.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_53.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_54.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_55.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_56.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_57.json, f1: 0.857, kappa: 0.850, weighted-F1: 0.857
file: checkpoint_58.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_59.json, f1: 0.855, kappa: 0.843, weighted-F1: 0.855
file: checkpoint_60.json, f1: 0.854, kappa: 0.843, weighted-F1: 0.854
file: checkpoint_61.json, f1: 0.854, kappa: 0.844, weighted-F1: 0.854
file: checkpoint_62.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_63.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_64.json, f1: 0.856, kappa: 0.848, weighted-F1: 0.856
file: checkpoint_65.json, f1: 0.856, kappa: 0.845, weighted-F1: 0.856
file: checkpoint_66.json, f1: 0.853, kappa: 0.845, weighted-F1: 0.853
file: checkpoint_67.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_68.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_69.json, f1: 0.855, kappa: 0.851, weighted-F1: 0.855
file: checkpoint_70.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_71.json, f1: 0.854, kappa: 0.845, weighted-F1: 0.854
file: checkpoint_72.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_73.json, f1: 0.854, kappa: 0.848, weighted-F1: 0.854
file: checkpoint_74.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_75.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_76.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_77.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_78.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_79.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_80.json, f1: 0.856, kappa: 0.848, weighted-F1: 0.856
file: checkpoint_81.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_82.json, f1: 0.855, kappa: 0.852, weighted-F1: 0.855
file: checkpoint_83.json, f1: 0.855, kappa: 0.853, weighted-F1: 0.855
file: checkpoint_84.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_85.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_86.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_87.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_88.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_89.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_90.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_91.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_92.json, f1: 0.856, kappa: 0.848, weighted-F1: 0.856
file: checkpoint_93.json, f1: 0.852, kappa: 0.843, weighted-F1: 0.852
file: checkpoint_94.json, f1: 0.856, kappa: 0.847, weighted-F1: 0.856
file: checkpoint_95.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_96.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_97.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_98.json, f1: 0.856, kappa: 0.844, weighted-F1: 0.856
file: checkpoint_99.json, f1: 0.853, kappa: 0.848, weighted-F1: 0.853
file: checkpoint_100.json, f1: 0.856, kappa: 0.848, weighted-F1: 0.856
file: checkpoint_101.json, f1: 0.856, kappa: 0.852, weighted-F1: 0.856
file: checkpoint_102.json, f1: 0.855, kappa: 0.844, weighted-F1: 0.855
file: checkpoint_103.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_104.json, f1: 0.857, kappa: 0.850, weighted-F1: 0.857
file: checkpoint_105.json, f1: 0.852, kappa: 0.846, weighted-F1: 0.852
file: checkpoint_106.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_107.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_108.json, f1: 0.856, kappa: 0.848, weighted-F1: 0.856
file: checkpoint_109.json, f1: 0.857, kappa: 0.854, weighted-F1: 0.857
file: checkpoint_110.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_111.json, f1: 0.857, kappa: 0.850, weighted-F1: 0.857
file: checkpoint_112.json, f1: 0.856, kappa: 0.845, weighted-F1: 0.856
file: checkpoint_113.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_114.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_115.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_116.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_117.json, f1: 0.853, kappa: 0.846, weighted-F1: 0.853
file: checkpoint_118.json, f1: 0.853, kappa: 0.844, weighted-F1: 0.853
file: checkpoint_119.json, f1: 0.855, kappa: 0.851, weighted-F1: 0.855
file: checkpoint_120.json, f1: 0.855, kappa: 0.845, weighted-F1: 0.855
file: checkpoint_121.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_122.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_123.json, f1: 0.855, kappa: 0.845, weighted-F1: 0.855
file: checkpoint_124.json, f1: 0.856, kappa: 0.846, weighted-F1: 0.856
file: checkpoint_125.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_126.json, f1: 0.854, kappa: 0.845, weighted-F1: 0.854
file: checkpoint_127.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_128.json, f1: 0.856, kappa: 0.847, weighted-F1: 0.856
file: checkpoint_129.json, f1: 0.857, kappa: 0.849, weighted-F1: 0.857
file: checkpoint_130.json, f1: 0.856, kappa: 0.845, weighted-F1: 0.856
file: checkpoint_131.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_132.json, f1: 0.855, kappa: 0.845, weighted-F1: 0.855
file: checkpoint_133.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_134.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_135.json, f1: 0.854, kappa: 0.845, weighted-F1: 0.854
file: checkpoint_136.json, f1: 0.854, kappa: 0.846, weighted-F1: 0.854
file: checkpoint_137.json, f1: 0.857, kappa: 0.852, weighted-F1: 0.857
file: checkpoint_138.json, f1: 0.857, kappa: 0.850, weighted-F1: 0.857
file: checkpoint_139.json, f1: 0.854, kappa: 0.849, weighted-F1: 0.854
file: checkpoint_140.json, f1: 0.854, kappa: 0.846, weighted-F1: 0.854
file: checkpoint_141.json, f1: 0.857, kappa: 0.845, weighted-F1: 0.857
file: checkpoint_142.json, f1: 0.856, kappa: 0.853, weighted-F1: 0.856
file: checkpoint_143.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_144.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_145.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_146.json, f1: 0.855, kappa: 0.850, weighted-F1: 0.855
file: checkpoint_147.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_148.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_149.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_150.json, f1: 0.857, kappa: 0.847, weighted-F1: 0.857
file: checkpoint_151.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_152.json, f1: 0.854, kappa: 0.845, weighted-F1: 0.854
file: checkpoint_153.json, f1: 0.854, kappa: 0.843, weighted-F1: 0.854
file: checkpoint_154.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_155.json, f1: 0.855, kappa: 0.844, weighted-F1: 0.855
file: checkpoint_156.json, f1: 0.854, kappa: 0.844, weighted-F1: 0.854
file: checkpoint_157.json, f1: 0.856, kappa: 0.846, weighted-F1: 0.856
file: checkpoint_158.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_159.json, f1: 0.855, kappa: 0.852, weighted-F1: 0.855
file: checkpoint_160.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_161.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_162.json, f1: 0.857, kappa: 0.855, weighted-F1: 0.857
file: checkpoint_163.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_164.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_165.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_166.json, f1: 0.855, kappa: 0.845, weighted-F1: 0.855
file: checkpoint_167.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_168.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_169.json, f1: 0.855, kappa: 0.844, weighted-F1: 0.855
file: checkpoint_170.json, f1: 0.856, kappa: 0.854, weighted-F1: 0.856
file: checkpoint_171.json, f1: 0.857, kappa: 0.848, weighted-F1: 0.857
file: checkpoint_172.json, f1: 0.854, kappa: 0.843, weighted-F1: 0.854
file: checkpoint_173.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_174.json, f1: 0.856, kappa: 0.851, weighted-F1: 0.856
file: checkpoint_175.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_176.json, f1: 0.855, kappa: 0.851, weighted-F1: 0.855
file: checkpoint_177.json, f1: 0.856, kappa: 0.850, weighted-F1: 0.856
file: checkpoint_178.json, f1: 0.855, kappa: 0.848, weighted-F1: 0.855
file: checkpoint_179.json, f1: 0.854, kappa: 0.844, weighted-F1: 0.854
file: checkpoint_180.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_181.json, f1: 0.855, kappa: 0.849, weighted-F1: 0.855
file: checkpoint_182.json, f1: 0.856, kappa: 0.852, weighted-F1: 0.856
file: checkpoint_183.json, f1: 0.857, kappa: 0.850, weighted-F1: 0.857
file: checkpoint_184.json, f1: 0.854, kappa: 0.842, weighted-F1: 0.854
file: checkpoint_185.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_186.json, f1: 0.854, kappa: 0.844, weighted-F1: 0.854
file: checkpoint_187.json, f1: 0.857, kappa: 0.847, weighted-F1: 0.857
file: checkpoint_188.json, f1: 0.854, kappa: 0.847, weighted-F1: 0.854
file: checkpoint_189.json, f1: 0.855, kappa: 0.847, weighted-F1: 0.855
file: checkpoint_190.json, f1: 0.856, kappa: 0.847, weighted-F1: 0.856
file: checkpoint_191.json, f1: 0.857, kappa: 0.847, weighted-F1: 0.857
file: checkpoint_192.json, f1: 0.855, kappa: 0.845, weighted-F1: 0.855
file: checkpoint_193.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_194.json, f1: 0.857, kappa: 0.851, weighted-F1: 0.857
file: checkpoint_195.json, f1: 0.853, kappa: 0.845, weighted-F1: 0.853
file: checkpoint_196.json, f1: 0.855, kappa: 0.846, weighted-F1: 0.855
file: checkpoint_197.json, f1: 0.856, kappa: 0.847, weighted-F1: 0.856
file: checkpoint_198.json, f1: 0.856, kappa: 0.849, weighted-F1: 0.856
file: checkpoint_199.json, f1: 0.853, kappa: 0.846, weighted-F1: 0.853
file: checkpoint_200.json, f1: 0.856, kappa: 0.847, weighted-F1: 0.856
Out[18]:
([0.85642047525751741, 0.85481068984542918, 0.85558667523667831],
 [array([[899,   2,  25,  10,   6,   0,   4,   5,  38,  11],
         [ 11, 952,   1,   2,   0,   1,   1,   4,  14,  14],
         [ 50,   0, 805,  34,  34,  50,  23,   2,   2,   0],
         [ 21,   2,  36, 797,  26,  66,  30,   9,   9,   4],
         [  8,   1,  46,  30, 841,  13,  42,  18,   1,   0],
         [  6,   1,  41, 170,  25, 726,   5,  25,   0,   1],
         [ 14,   3,  35,  33,   4,   4, 903,   3,   0,   1],
         [ 10,   0,  20,  30,  44,  20,   1, 873,   0,   2],
         [ 32,  10,   5,  13,   4,   1,   2,   0, 928,   5],
         [ 19,  95,   3,   7,   3,   1,   3,   1,  30, 838]]),
  array([[905,   2,  25,  10,   6,   0,   4,   6,  36,   6],
         [ 13, 947,   1,   6,   0,   1,   1,   3,  16,  12],
         [ 49,   0, 809,  34,  34,  45,  25,   2,   2,   0],
         [ 22,   1,  35, 798,  30,  64,  31,   6,   9,   4],
         [  8,   1,  44,  28, 847,  13,  44,  14,   1,   0],
         [  5,   0,  42, 175,  27, 722,   7,  22,   0,   0],
         [ 14,   2,  34,  35,   4,   3, 905,   2,   0,   1],
         [ 10,   0,  21,  32,  55,  22,   2, 856,   0,   2],
         [ 36,   7,   5,  14,   4,   1,   2,   0, 927,   4],
         [ 31,  87,   3,  10,   2,   1,   5,   1,  32, 828]]),
  array([[900,   2,  26,  11,   6,   0,   4,   5,  35,  11],
         [ 12, 952,   1,   2,   0,   1,   1,   4,  14,  13],
         [ 47,   0, 810,  36,  34,  44,  26,   2,   1,   0],
         [ 19,   2,  35, 802,  28,  64,  32,   6,   9,   3],
         [  8,   1,  43,  29, 843,  13,  48,  14,   1,   0],
         [  5,   0,  41, 178,  26, 719,   7,  24,   0,   0],
         [ 14,   2,  31,  35,   4,   3, 908,   2,   0,   1],
         [  9,   1,  20,  33,  56,  21,   2, 856,   0,   2],
         [ 35,  10,   5,  18,   4,   1,   2,   0, 921,   4],
         [ 19,  94,   3,   8,   2,   1,   5,   1,  26, 841]])],
 ['checkpoint_190.json', 'checkpoint_193.json', 'checkpoint_200.json'])